{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic 回归——皮马印第安人糖尿病分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "#评价指标logloss\n",
    "from sklearn.metrics import log_loss  \n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 768 non-null int64\n",
      "Glucose                     768 non-null int64\n",
      "BloodPressure               768 non-null int64\n",
      "SkinThickness               768 non-null int64\n",
      "Insulin                     768 non-null int64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null int64\n",
      "Outcome                     768 non-null int64\n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>120.894531</td>\n",
       "      <td>69.105469</td>\n",
       "      <td>20.536458</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>31.992578</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>31.972618</td>\n",
       "      <td>19.355807</td>\n",
       "      <td>15.952218</td>\n",
       "      <td>115.244002</td>\n",
       "      <td>7.884160</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.300000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>30.500000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Pregnancies     Glucose  BloodPressure  SkinThickness     Insulin  \\\n",
       "count   768.000000  768.000000     768.000000     768.000000  768.000000   \n",
       "mean      3.845052  120.894531      69.105469      20.536458   79.799479   \n",
       "std       3.369578   31.972618      19.355807      15.952218  115.244002   \n",
       "min       0.000000    0.000000       0.000000       0.000000    0.000000   \n",
       "25%       1.000000   99.000000      62.000000       0.000000    0.000000   \n",
       "50%       3.000000  117.000000      72.000000      23.000000   30.500000   \n",
       "75%       6.000000  140.250000      80.000000      32.000000  127.250000   \n",
       "max      17.000000  199.000000     122.000000      99.000000  846.000000   \n",
       "\n",
       "              BMI  DiabetesPedigreeFunction         Age     Outcome  \n",
       "count  768.000000                768.000000  768.000000  768.000000  \n",
       "mean    31.992578                  0.471876   33.240885    0.348958  \n",
       "std      7.884160                  0.331329   11.760232    0.476951  \n",
       "min      0.000000                  0.078000   21.000000    0.000000  \n",
       "25%     27.300000                  0.243750   24.000000    0.000000  \n",
       "50%     32.000000                  0.372500   29.000000    0.000000  \n",
       "75%     36.600000                  0.626250   41.000000    1.000000  \n",
       "max     67.100000                  2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## 各属性的统计特性\n",
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAE7tJREFUeJzt3X20ZXV93/H3BwZ8jsPDhEWZmQ5WTJYuFcloIaY2Yh5EW8ayBGNZMqGzOsmqtaSkiRMbpY010SZIpDYkswJhyKIQoCKj0igZHmysUGZQQR4s4xSYmSoMhkcpWODbP87v6mGy5949D+eew73v11pnnb1/Z+99vsc13I97//b+/VJVSJK0s/3GXYAkaTIZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOi0YdwF749BDD61ly5aNuwxJel7ZtGnTg1W1aKbtntcBsWzZMjZu3DjuMiTpeSXJvX228xKTJKmTASFJ6mRASJI6GRCSpE4GhCSp00gDIsk9SW5L8vUkG1vbwUmuSXJ3ez+otSfJuUk2J7k1yTGjrE2SNL3ZOIN4a1UdXVXL2/oaYENVHQVsaOsAJwBHtddq4LxZqE2StAvjuMS0AljXltcB7xpqv6gGbgQWJjl8DPVJkhh9QBTwpSSbkqxubYdV1Xfa8neBw9ryEcDWoX23tTZJ0hiM+knqn6mq7Ul+HLgmyV3DH1ZVJandOWALmtUAS5cu3esCf+o3LtrrY2ju2fT7p427BGnsRnoGUVXb2/sDwJXAm4D7py4dtfcH2ubbgSVDuy9ubTsfc21VLa+q5YsWzTiUiCRpD40sIJK8JMnLppaBXwC+CawHVrbNVgJXteX1wGntbqZjgUeGLkVJkmbZKC8xHQZcmWTqe/5LVf1lkpuBy5KsAu4FTmnbXw28A9gMPAGcPsLaJEkzGFlAVNUW4PUd7d8D3tbRXsD7R1WPJGn3+CS1JKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROIw+IJPsn+VqSz7f1I5PclGRzkr9IcmBrf0Fb39w+Xzbq2iRJuzYbZxBnAHcOrX8COKeqXgk8BKxq7auAh1r7OW07SdKYjDQgkiwG3gn8aVsPcDxwRdtkHfCutryirdM+f1vbXpI0BqM+g/hD4DeBZ9v6IcDDVfV0W98GHNGWjwC2ArTPH2nbS5LGYMaASPKSJPu15VclOTHJAT32+0fAA1W1aR/UOXzc1Uk2Jtm4Y8eOfXloSdKQPmcQXwZemOQI4EvA+4ALe+z3ZuDEJPcAlzK4tPQpYGGSBW2bxcD2trwdWALQPn858L2dD1pVa6tqeVUtX7RoUY8yJEl7ok9ApKqeAE4C/qiqTgZeM9NOVfVbVbW4qpYBvwRcW1WnAtcB726brQSuasvr2zrt82urqnr/EknSPtUrIJIcB5wKfKG17b8X3/lB4Mwkmxn0MZzf2s8HDmntZwJr9uI7JEl7acHMm/BrwG8BV1bV7UleweAsoLequh64vi1vAd7Usc2TwMm7c1xJ0ujMGBBVdQNwQ5IXt/UtwL8adWGSpPHqcxfTcUnuAO5q669P8kcjr0ySNFZ9+iD+EPhF2h1FVfUN4C2jLEqSNH69HpSrqq07NT0zglokSROkTyf11iQ/DVR7QG7nsZUkSXNQnzOIXwXez2AojO3A0W1dkjSH9bmL6UEGz0BIkuaRPncxrUuycGj9oCQXjLYsSdK49bnE9LqqenhqpaoeAt4wupIkSZOgT0Dsl+SgqZUkB9Ovc1uS9DzW5w/92cBXk1wOhMFAeh8baVWSpLHr00l9UZJNwFtb00lVdcdoy5IkjVvfS0V3MZg/egFAkqVVdd/IqpIkjd2MAZHkA8BZwP0MnqAOUMDrRluaJGmc+pxBnAH8RFX9rdndJElzV5+7mLYCj4y6EEnSZOlzBrEFuD7JF4Cnphqr6pMjq0qSNHZ9AuK+9jqwvSRJ80Cf21z/PUCSF1fVE6MvSZI0CZxRTpLUyRnlJEmdnFFOktTJGeUkSZ2cUU6S1GnaM4gk+wPvqypnlJOkeWbaM4iqegb4p7NUiyRpgvTpg/jrJJ8G/gL4/lRjVd0ysqokSWPXJyCObu+/M9RWwPH7vhxJ0qSYqQ9iP+C8qrpsluqRJE2ImfogngV+c5ZqkSRNkD63uf5Vkn+TZEmSg6deI69MkjRWffog3tPeh599KOAV+74cSdKk6DOa65GzUYgkabL0mZP6tK72qrpohv1eCHwZeEH7niuq6qwkRwKXAocAmxg8iPeDJC8ALgJ+isHAgO+pqnt247dIkvahPn0Qbxx6/QPg3wEn9tjvKeD4qno9g1tl357kWOATwDlV9UrgIWBV234V8FBrP6dtJ0kakz6XmD4wvJ5kIYMzgJn2K+DxtnpAe009PzH1dPY6BoFzHrCiLQNcAXw6SdpxJEmzrNdw3zv5PtCrXyLJ/km+DjwAXAN8G3i4qp5um2xjMAgg7X0rQPv8EQaXoSRJY9CnD+JzDP6fPwwC5dVArwfn2lhOR7ezjiuBn9zDOofrWQ2sBli6dOneHk6StAt9bnP9g6Hlp4F7q2rb7nxJVT2c5DrgOGBhkgXtLGExgyHEae9LgG1JFgAvp81it9Ox1gJrAZYvX+7lJ0kakT6XmO4DbqqqG6rqK8D3kiybaacki9qZA0leBPw8g4mGrgPe3TZbCVzVlte3ddrn19r/IEnj0ycgLgeeHVp/prXN5HDguiS3AjcD11TV54EPAmcm2cygj+H8tv35wCGt/UxgTb+fIEkahT6XmBZU1Q+mVtozCwfOtFNV3Qq8oaN9C/CmjvYngZN71CNJmgV9ziB2JPnhcw9JVgAPjq4kSdIk6HMG8avAxW3SIBjcmtr5dLUkae7o86Dct4Fjk7y0rT8+wy6SpDlgxktMSX43ycKqeryqHk9yUJL/MBvFSZLGp08fxAlV9fDUSlU9BLxjdCVJkiZBn4DYv420CvzwmYYXTLO9JGkO6NNJfTGwIcmftfXTGQyyJ0maw/p0Un8iyTeAn2tNH62qL462LEnSuPU5gwD4Gj8arvtroytH0pT7fue14y5BE2jpR26bte/qcxfTKcD/ZDA+0inATUnePf1ekqTnuz5nEP8WeGNVPQCDQfiAv2IwqY8kaY7qcxfTflPh0Hyv536SpOexPmcQf5nki8Albf09wNWjK0mSNAn63MX0G0lOAn6mNa2tqitHW5Ykadx63cVUVZ8BPjPiWiRJE8S+BElSJwNCktRplwGRZEN7/8TslSNJmhTT9UEcnuSngROTXApk+MOqumWklUmSxmq6gPgI8GFgMfDJnT4r4PhRFSVJGr9dBkRVXQFckeTDVfXRWaxJkjQB+jwH8dEkJwJvaU3XV9XnR1uWJGnc+gzW93vAGcAd7XVGkt8ddWGSpPHq86DcO4Gjq+pZgCTrGAz5/aFRFiZJGq++z0EsHFp++SgKkSRNlj5nEL8HfC3JdQxudX0LsGakVUmSxq5PJ/UlSa4H3tiaPlhV3x1pVZKkses7WN93gPUjrkWSNEEci0mS1MmAkCR1mjYgkuyf5K7ZKkaSNDmmDYiqegb4VpKls1SPJGlC9LnEdBBwe5INSdZPvWbaKcmSJNcluSPJ7UnOaO0HJ7kmyd3t/aDWniTnJtmc5NYkx+zdT5Mk7Y0+dzF9eA+P/TTw61V1S5KXAZuSXAP8MrChqj6eZA2DZyo+CJwAHNVefx84r71LksZgxjOIqroBuAc4oC3fDMw4F0RVfWdqzoiqegy4EzgCWAGsa5utA97VllcAF9XAjcDCJIfv3s+RJO0rfQbr++fAFcCftKYjgM/uzpckWQa8AbgJOKw9VwHwXeCwoeNuHdptW2uTJI1Bnz6I9wNvBh4FqKq7gR/v+wVJXgr8V+DXqurR4c+qqhhMPtRbktVJNibZuGPHjt3ZVZK0G/oExFNV9YOplSQL6PlHPckBDMLh4qr6TGu+f+rSUXt/oLVvB5YM7b64tT1HVa2tquVVtXzRokV9ypAk7YE+AXFDkg8BL0ry88DlwOdm2ilJgPOBO6tqeMrS9cDKtrwSuGqo/bR2N9OxwCNDl6IkSbOsz11Ma4BVwG3ArwBXA3/aY783A+8Dbkvy9db2IeDjwGVJVgH3Aqe0z64G3gFsBp4ATu/5GyRJI9BnNNdn2yRBNzG4tPSt1ncw035/zWB48C5v69i+GPR3SJImwIwBkeSdwB8D32bwB//IJL9SVf9t1MVJksanzyWms4G3VtVmgCR/D/gCYEBI0hzWp5P6salwaLYAj42oHknShNjlGUSSk9rixiRXA5cx6IM4mcHT1JKkOWy6S0z/eGj5fuAftuUdwItGVpEkaSLsMiCqyttMJWke63MX05HAB4Blw9tX1YmjK0uSNG597mL6LIMnoj8HPDvaciRJk6JPQDxZVeeOvBJJ0kTpExCfSnIW8CXgqanGqbkeJElzU5+AeC2DMZWO50eXmKqtS5LmqD4BcTLwiuEhvyVJc1+fJ6m/CSwcdSGSpMnS5wxiIXBXkpt5bh+Et7lK0hzWJyDOGnkVkqSJ02c+iBtmoxBJ0mTp8yT1Y/xoDuoDgQOA71fVj42yMEnSePU5g3jZ1HKbZ3oFcOwoi5IkjV+fu5h+qAY+C/ziiOqRJE2IPpeYThpa3Q9YDjw5sookSROhz11Mw/NCPA3cw+AykyRpDuvTB+G8EJI0D0035ehHptmvquqjI6hHkjQhpjuD+H5H20uAVcAhgAEhSXPYdFOOnj21nORlwBnA6cClwNm72k+SNDdM2weR5GDgTOBUYB1wTFU9NBuFSZLGa7o+iN8HTgLWAq+tqsdnrSpJ0thN96DcrwN/B/ht4P8kebS9Hkvy6OyUJ0kal+n6IHbrKWtJ0txiCEiSOhkQkqROBoQkqZMBIUnqNLKASHJBkgeSfHOo7eAk1yS5u70f1NqT5Nwkm5PcmuSYUdUlSepnlGcQFwJv36ltDbChqo4CNrR1gBOAo9prNXDeCOuSJPUwsoCoqi8Df7NT8woGT2TT3t811H5Rm5DoRmBhksNHVZskaWaz3QdxWFV9py1/FzisLR8BbB3abltr+1uSrE6yMcnGHTt2jK5SSZrnxtZJXVUF1B7st7aqllfV8kWLFo2gMkkSzH5A3D916ai9P9DatwNLhrZb3NokSWMy2wGxHljZllcCVw21n9buZjoWeGToUpQkaQz6zEm9R5JcAvwscGiSbcBZwMeBy5KsAu4FTmmbXw28A9gMPMFg3glJ0hiNLCCq6r27+OhtHdsW8P5R1SJJ2n0+SS1J6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6mRASJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqdNEBUSStyf5VpLNSdaMux5Jms8mJiCS7A/8Z+AE4NXAe5O8erxVSdL8NTEBAbwJ2FxVW6rqB8ClwIox1yRJ89YkBcQRwNah9W2tTZI0BgvGXcDuSrIaWN1WH0/yrXHWM8ccCjw47iImQf5g5bhL0HP5b3PKWdkXR/m7fTaapIDYDiwZWl/c2p6jqtYCa2erqPkkycaqWj7uOqSd+W9zPCbpEtPNwFFJjkxyIPBLwPox1yRJ89bEnEFU1dNJ/iXwRWB/4IKqun3MZUnSvDUxAQFQVVcDV4+7jnnMS3eaVP7bHINU1bhrkCRNoEnqg5AkTRADQg5xoomV5IIkDyT55rhrmY8MiHnOIU404S4E3j7uIuYrA0IOcaKJVVVfBv5m3HXMVwaEHOJEUicDQpLUyYBQryFOJM0/BoQc4kRSJwNinquqp4GpIU7uBC5ziBNNiiSXAF8FfiLJtiSrxl3TfOKT1JKkTp5BSJI6GRCSpE4GhCSpkwEhSepkQEiSOhkQmveSLE5yVZK7k3w7yafaMyHT7fOh2apPGhcDQvNakgCfAT5bVUcBrwJeCnxshl0NCM15BoTmu+OBJ6vqzwCq6hngXwP/LMm/SPLpqQ2TfD7Jzyb5OPCiJF9PcnH77LQktyb5RpI/b23Lklzb2jckWdraL0xyXpIbk2xpx7wgyZ1JLhz6vl9I8tUktyS5PMlLZ+1/FQkDQnoNsGm4oaoeBe5jF3O2V9Ua4P9W1dFVdWqS1wC/DRxfVa8Hzmib/idgXVW9DrgYOHfoMAcBxzEIo/XAOa2W1yY5Osmh7Zg/V1XHABuBM/fFD5b66vwPQNJuOR64vKoeBKiqqfkLjgNOast/DvzHoX0+V1WV5Dbg/qq6DSDJ7cAyBoMmvhr4yuAqGAcyGHJCmjUGhOa7O4B3Dzck+TFgKfAwzz3LfuE+/N6n2vuzQ8tT6wuAZ4Brquq9+/A7pd3iJSbNdxuAFyc5DX44BevZDKa63AIcnWS/JEsYzL435f8lOaAtXwucnOSQdoyDW/v/YDA6LsCpwH/fjbpuBN6c5JXtmC9J8qrd/XHS3jAgNK/VYLTKf8LgD/zdwP8CnmRwl9JXgP/N4CzjXOCWoV3XArcmubiNfvsx4IYk3wA+2bb5AHB6kluB9/Gjvok+de0Afhm4pO3/VeAn9/R3SnvC0VwlSZ08g5AkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1On/A09qIrpP2Ss7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Outcome 分布，看看两类样本分布是否均衡\n",
    "sns.countplot(train.Outcome);\n",
    "pyplot.xlabel('Outcome');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "两类样本不太均衡。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从数据中分离输入特征X_train和输出y_train\n",
    "y_train = train['Outcome']\n",
    "X_train = train.drop('Outcome', axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 对训练数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr= LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.49442383 0.50834194 0.4830274  0.43615578 0.48793522]\n",
      "cv logloss is: 0.4819768328671518\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print ('logloss of each fold is: ',-loss)\n",
    "print ('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正则化的 Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "#训练\n",
    "grid.fit(X_train,y_train)\n",
    "#预测\n",
    "# y_test_pred_grid = grid.predict(X_test)\n",
    "# y_train_pred_grid = grid.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00107422, 0.00115356, 0.0010046 , 0.00117035, 0.00118651,\n",
       "        0.00113292, 0.00108452, 0.00120921, 0.00122666, 0.00119996,\n",
       "        0.0011107 , 0.00113416, 0.00123901, 0.00118523]),\n",
       " 'std_fit_time': array([4.13745054e-04, 1.38402558e-04, 1.44913127e-04, 2.73713725e-04,\n",
       "        1.03447247e-04, 1.32807358e-04, 5.34337260e-05, 1.81507073e-04,\n",
       "        1.74636491e-04, 1.47730970e-04, 1.35963040e-04, 2.41839801e-04,\n",
       "        2.62946074e-04, 1.84238080e-04]),\n",
       " 'mean_score_time': array([0.00069795, 0.00064917, 0.00070691, 0.00066667, 0.00053563,\n",
       "        0.00055585, 0.00053201, 0.00057054, 0.0005167 , 0.00058947,\n",
       "        0.00052242, 0.00050874, 0.00057831, 0.00054245]),\n",
       " 'std_score_time': array([2.21515655e-04, 1.50875934e-04, 2.38520553e-04, 9.71509810e-05,\n",
       "        8.43277583e-05, 5.77163271e-05, 7.25630539e-05, 8.28538009e-05,\n",
       "        5.41285581e-05, 1.17281160e-04, 4.65814993e-05, 4.18888349e-05,\n",
       "        1.05044055e-04, 5.40999863e-05]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
       "                    100, 1000, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'split0_test_score': array([-0.69314718, -0.63247716, -0.64643499, -0.51842729, -0.49335058,\n",
       "        -0.4893871 , -0.49423694, -0.49442383, -0.49550391, -0.49553449,\n",
       "        -0.49564739, -0.49565533, -0.49566261, -0.49566752]),\n",
       " 'split1_test_score': array([-0.69314718, -0.64153853, -0.65138433, -0.54263843, -0.5162119 ,\n",
       "        -0.50836483, -0.50929523, -0.50834194, -0.50874639, -0.50865995,\n",
       "        -0.50870921, -0.5086969 , -0.50870043, -0.50870065]),\n",
       " 'split2_test_score': array([-0.69314718, -0.63662203, -0.64570415, -0.52403018, -0.47823053,\n",
       "        -0.48374951, -0.48180197, -0.4830274 , -0.48313527, -0.48326216,\n",
       "        -0.48327949, -0.48329051, -0.48329238, -0.4832934 ]),\n",
       " 'split3_test_score': array([-0.69314718, -0.63189062, -0.64433664, -0.50307754, -0.45405618,\n",
       "        -0.44275292, -0.43661786, -0.43615578, -0.43569105, -0.43565059,\n",
       "        -0.43561177, -0.43560284, -0.43559905, -0.43559809]),\n",
       " 'split4_test_score': array([-0.69314718, -0.64067605, -0.64757466, -0.53287257, -0.49692195,\n",
       "        -0.48956528, -0.48810481, -0.48793522, -0.48809257, -0.48807935,\n",
       "        -0.48809991, -0.48809877, -0.48810345, -0.48810077]),\n",
       " 'mean_test_score': array([-0.69314718, -0.63664181, -0.6470899 , -0.52422544, -0.48778617,\n",
       "        -0.48280717, -0.48206253, -0.48202874, -0.48228681, -0.48229036,\n",
       "        -0.48232271, -0.48232204, -0.48232476, -0.48232527]),\n",
       " 'std_test_score': array([1.11022302e-16, 4.00318135e-03, 2.39292199e-03, 1.33688676e-02,\n",
       "        2.07322065e-02, 2.16460757e-02, 2.44366086e-02, 2.44134571e-02,\n",
       "        2.47954013e-02, 2.47956827e-02, 2.48339374e-02, 2.48355509e-02,\n",
       "        2.48387349e-02, 2.48395553e-02]),\n",
       " 'rank_test_score': array([14, 12, 13, 11, 10,  9,  2,  1,  3,  4,  6,  5,  7,  8],\n",
       "       dtype=int32),\n",
       " 'split0_train_score': array([-0.69314718, -0.63735556, -0.64699594, -0.5192309 , -0.47577709,\n",
       "        -0.46925194, -0.46602427, -0.46594583, -0.46589353, -0.46589258,\n",
       "        -0.46589203, -0.46589201, -0.46589201, -0.46589201]),\n",
       " 'split1_train_score': array([-0.69314718, -0.63310132, -0.63649244, -0.51411049, -0.47224315,\n",
       "        -0.46704903, -0.46420466, -0.46410775, -0.46406256, -0.4640616 ,\n",
       "        -0.46406112, -0.46406111, -0.4640611 , -0.4640611 ]),\n",
       " 'split2_train_score': array([-0.69314718, -0.6356616 , -0.65056852, -0.51951737, -0.4788676 ,\n",
       "        -0.47222043, -0.46932983, -0.46925675, -0.46921155, -0.46921058,\n",
       "        -0.4692101 , -0.46921009, -0.46921008, -0.46921008]),\n",
       " 'split3_train_score': array([-0.69314718, -0.63804443, -0.65182756, -0.52743851, -0.48993936,\n",
       "        -0.48357524, -0.48097754, -0.48088442, -0.48084376, -0.48084284,\n",
       "        -0.48084241, -0.4808424 , -0.4808424 , -0.48084239]),\n",
       " 'split4_train_score': array([-0.69314718, -0.63358836, -0.646171  , -0.51771935, -0.47759523,\n",
       "        -0.47164861, -0.468824  , -0.46875047, -0.46870588, -0.46870513,\n",
       "        -0.46870465, -0.46870464, -0.46870464, -0.46870464]),\n",
       " 'mean_train_score': array([-0.69314718, -0.63555025, -0.64641109, -0.51960333, -0.47888449,\n",
       "        -0.47274905, -0.46987206, -0.46978904, -0.46974346, -0.46974255,\n",
       "        -0.46974206, -0.46974205, -0.46974205, -0.46974205]),\n",
       " 'std_train_score': array([0.        , 0.00196666, 0.00539262, 0.00436479, 0.00596047,\n",
       "        0.00571694, 0.00585916, 0.0058569 , 0.00585954, 0.00585954,\n",
       "        0.00585957, 0.00585957, 0.00585957, 0.00585957])}"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4820287373783356\n",
      "{'C': 1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "正则函数选用L2，正则参数C调为0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/zcz/.local/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8FHX++PHXe9MJCSWBUAKEKgKhRgEBFURBRIoUvbMdFk5/FsTzzn6285SvpwLiiQgCKkoERLGD2AAPlRqaSi+BkBCSkJC++/n9sRsIkJBNspvZJO/n47HuzM5nZt4jyb7zmc/Me8QYg1JKKVVZNqsDUEopVTNoQlFKKeURmlCUUkp5hCYUpZRSHqEJRSmllEdoQlFKKeURmlCUUkp5hCYUpZRSHqEJRSmllEf4W7FTEWkIxAMxwD5gvDEmrZS24cB24GNjzL2uz74HmgI5rmZXGWOSy9pvZGSkiYmJqWT0SilVu6xfv/6YMaZRWe0sSSjAI8BKY8yLIvKIa/7hUto+B/xYwuc3GmPWlWenMTExrFtXrlWUUqrWE5H97rSz6pTXSGC+a3o+MKqkRiLSC4gClldRXEoppSrIqoQSZYw54ppOwpk0ziAiNuBl4KFStjFXRDaJyJMiIl6KUymllJu8dspLRL4BmpSw6PHiM8YYIyIllTz+f8AXxphDJeSLG40xiSISBiwBbgbeKSWOicBEgJYtW5bvIJRSSrnNawnFGDO4tGUiclREmhpjjohIU6CkAfW+wAAR+X9AXSBQRLKMMY8YYxJd+8gUkfeBiykloRhjZgGzAOLi4rRWv1JKeYlVp7yWAbe6pm8FPjm7gTHmRmNMS2NMDM7TXu8YYx4REX8RiQQQkQBgOLC1asJWSilVGqsSyovAlSKyExjsmkdE4kRkdhnrBgFfi0gCsAlIBN7yZrBKKaXKJrXpiY1xcXFGLxtWSqnyEZH1xpi4strpnfJKKaU8QhOKUhbrPXcMveeOsToMj9Bj8T1VeRyaUJRSSnmEJhSllFIeoQlFVUsTvprAhK8mWB2GUqoYTShKKaU8QhOKUkopj7CqfL1SyuWpBTucEzXgDJ4ei++pyuPQHopSSimP0B6KqpZGvJzgnBhqbRxKqdM0oShlAWO348jOxpGdTWChIdDfTu7P34IxgLMc0umySK53Y05NnpoHwHHWPOBwnLVu0X/MmW3PfsecOe1aZkyx+VP7d5zRDAwRfvkAZH/5XukHX03UlGMpOg5HRiq2ehFe3ZcmFKXKYIzB5Oc7E8DJkzhOut6zS3kv7bOsEzhOZuHIzsHkF57afnMA/Nh76z1WHaLHhBMAwP7Jz1scSeXVlGMpOo6CnZsJihvk1X1pQlE1jnE4cGTnuL7QXQkgu5Qv+jPes0+3P2sZhYVl7xjAZsMWEoQtyB9boGCzFWKz5RFADjb/QmwRBlsTgy0oAFu9CGwNoji6fSsFDhsxo250bcT1QDmRs6Y59/Ni+z2HnL2uONcqNn/28lNvYjvjI9eHZ82fu+7vb08F4ILbJ58bTzXz+5xXgep/LEXHEdCuq9f3pQlFVUtB+Q4iThSw76abzkkKJjvb7e1IUBC20FBsdeqcevcLDyegSZNzPreFhjpfgTZs9gxsBcew5SVjy0nEdvIAtsy9SEHG6e9YWwA0bA0RXSGiLUS0O/2q2/jUl/H/BncCIPbO5zz9v6nKHX3rDQB6jb/X4kgq7+is/wLV/1iKjsNWP9Lr+9KEoqqdjGXLaJWcixEQsRHQOOrML/xT03VKTgrF3sW/lF8BewGk7YfUXa7XDuf73t2QefjMtuHRENkO2o0rljTaQr2W4Ke/Yqr20J92VW048vM5+sILpH+wkLxASImAIe+W+ORn9xgDJw4XSxq7T0+n7QNHsdNcwfUhsj20ufzM3kbDNhBYp5JHplTNoAlFVQsFhw9z6IHJ5CYk0PC221j/9dtnjiucT076mcmieAIpOHm6nX8wNGwLUZ2h08gzT1HVaeidAwPmjQ8EYJjX9lB1nrnxQkCPxZdU5XFoQlE+L2vNGg7/7SFMQQHNp08j/KqrYPncMxsV5sHxPecmjNRdcDLldDuxQf2WENEeWvU7s7cR3rzkwW0v2x/Qtsr3qZQ3aEJRPss4HKS++SYp018jqF07mk+fRlDr1gDYbIZ2LU7Au9c5k0b6Ac64T6JulDNJXHD1mT2NBjHgH2TJ8ShV02lCUT7JnpHB4X88TNYPPxB+7bU0feZpbHVOj1UcujqPoRlZzt5H9EXQ7U+nB8Mj2kJwPQujV6p20oSifE7Otm0k3j+JguRkov75JA3+9CfkrPGSAdlZ7PcPoNVdqyyKUil1Nk0oyqekL15M0rPP4dewITHvvUtIt27nNkrdTfuCPP6vQST9E1fjMA4KHYU4jAO7sWN32LEb++l5Y8fhKDZdWjvHmeucPV/ieiXs6+z1Cux28gsLybc7X4UO+6mX3dgpcBQAEPt2H9eth8VvXCx+U2Lxz4RTNyqe9bmU2ubc9nLq81K2Iaenz257KtZibTLz8gDoP++mEv99z9z/OQvOS87b6PwbKGtpScuzcp3/LpfPr97lhouOY2vSAbo0aenVfWlCUT7BkZtL0nPPkbHkI0Iv6Uuz//wH/4YlX1l1eONcnmjSmF9Dglnwzd0e2b9NbNjEhp/4nXrZbKfnTy2zOecFG8bYcBjB4QC7Q7DbhUKHUGB33lif73oZY8OYYMAGRgAbNvEjJMCfOv7+pOWlgThoXa+5q9qW41TtLOP6xBhTVOHL+Zkxp5YW1egymLPWc26LEj8/vd3i85y9TWOc38bG4CjaFmduq6j+mAHElgtAjuO42//vTQlT58yeNyMU24I5//JzyHkW+ztrYGUU5p5v577P35lQTubneX9XXt+DUmXIP3iQQ5Mmkbd9BxF3/ZVG992H+Pmd084Yw8e7PmbKoU+wBwUSVSC8PPLdM7/wz0oEp5bZSk4MRfMigsNhSMvOJyUrj+QTeaRk5pGS5XxPzswjJTPX+VlmHidyzy3FIgIRoYFE1g2icXgwjRoE0Sjs9KtxsemwIP9Tp/F6zx0DwOc3TvXu/+gqUHQsP09YYnEklVdTjqXoOHq3bO/1fWlCUZbK/P57Dv/jYQCi3/gvYQMHltjuWM4xnvnfM3x/8Ht65eYSVdiQ70Oj6daohFNiZ8nOLzyVCFIys13J4XTCSHYlimNZ+dgd5/65WifQ71QyuKBJGP3bRToTRt0zk0XD0ED8/fQRQ6r20oSiLGHsdlJmzCD1jZkEXXgh0dOnEdiiRYltV+xfwXP/e46TBSd5KPQCbj64isFNO+EoDGJrYsapXsQ5r6w8kk/kcjLffs42bQKRdU8ng05Nw53Joah34ZpuFBZEaJD+mijlDv1NUVWuMC2Nw397iJM//US9MdfR5MknsQUHn9MuIy+DF355gc/3fE6niE78u+8ztJ19NVwwjLSUINIODGf4a6vPWCcs2P9UMujcLJyBFzQu8bRTgzqB+NncvNNeKeUWTSiqSuVs3syhByZjT02l6b+eo/7YsSW2+ynxJ5786UlSc1K5u9vd3Nn1TgJ2fgM5x6HbDeQs3oHYCnjjz31oFBZM47AgIusGERJ47tiLUqpqaEJRVcIYQ9oHH3D0hRcJaNyYVh+8T0jnzue0yy7I5pX1rxD/ezxt6rVh+sDpdI50tUuIhzoRZEVfSm5mDiHhuxnaZXQVH4lSqjSaUJTXObKzOfL005xY9imhl11K8ylT8Ktf/5x2m5I38fjqxzmYeZCbO93M/T3uJ9jfdSosNwN++wJ63crn21LABBBcb2cVH4lS6nw0oSivytu7l8T7J5G3axeR999H5F13IWcVYMy35/P6pteZt20eTUObMmfIHC5qctGZG9q+DOx50PUGFn16CL/AdAKCU1BK+Q5NKMprTqxYwZFHH0P8/Wnx1lvU7d/vnDa/H/+dR1c/ys60nYxpP4a/X/R3QgNCz91YQjw0bMuewA6s2/8jdSN3ul29XilVNTShKI8zhYUkv/oqx+e8TXBsLNHTphLQrNkZbQodhczbNo/XN71OvcB6zBg0g8taXFbyBtMPwr5VMPBxFm9IxM8mBNfbVQVHopQqD0sSiog0BOKBGGAfMN4Yk1ZCOzuwxTV7wBgzwvV5a2AhEAGsB242xuR7P3JVlsKUFBIf/BvZv/5K/T/dQNSjj2ILDDyjzf4T+3ls9WMkpCRwVaureKLPEzQIblD6RrcuBsDeZRxL3tzNZR0asc3kePMwlFIVYNVtvY8AK40x7YGVrvmS5BhjurteI4p9PgV41RjTDkgDbvduuMod2evXs/e6MeRs2UKzKS/S9KmnzkgmDuPg/R3vM3bZWPZl7GPKgCn857L/nD+ZGAOb46FFH348FsrRE3mMj4uugqNRSpWXVQllJDDfNT0fGOXuiuIsgDQIWFyR9ZXnGWNInTeP/bfcitQJISY+nnojR57RJulkEn9d8Vde+OUFejXpxdKRSxnWZtg5ZenPkbQFUnZA1/EsXneIhqGBDOoY5cWjUUpVlFVjKFHGmCOu6SSgtG+IYBFZBxQCLxpjPsZ5mivdGFNUne8Q0Nyr0apS2bNOcuSJJ8j86ivqDr6CZi+8gF9Y2Knlxhg+2/MZL/z8AoWmkCf7PMm4DuPKTiRFEuLBFkBGm2tY8fFGbuzTkkB/rZellC/yWkIRkW+AJiUserz4jDHGiEhpBaRbGWMSRaQN8K2IbAEyyhnHRGAiQMuW3n0WQG2Tt2sXh+6fRP6+fTR+6G80vP32MxLF8dzjPPu/Z1l5YCU9Gvfg+X7P0yK85HpdJXLYYcsi6DCEj3/PJd/uYFyvcqyvlKpSXksoxpjBpS0TkaMi0tQYc0REmgLJpWwj0fW+R0S+B3oAS4D6IuLv6qVEA4nniWMWMAsgLi7uPA9GUOWR8fnnHHnyn9hCQmg5dy6hvS8+Y/nKAyt59n/PkpmfyYO9HuSWTrfgZytnWZQ930PWUeg6nkUrD9K5WTidmoV77iCUUh5l1bmDZcCtrulbgU/ObiAiDUQkyDUdCfQDthvnE4G+A8aeb33lHSY/n6Tn/83hvz1EcMeOtP7oozOSSWZ+Jo+vfpwHvnuAqDpRxA+PZ0KXCeVPJgAJH0JwPXbU7cvWxBOMj9PeiVK+zKoxlBeBD0XkdmA/MB5AROKAu4wxdwAXAm+KiANn4nvRGLPdtf7DwEIR+RewEZhT1QdQGxUcPUriA5PJ2biRhrfeQuOHHkICAk4tX3tkLU+ueZKU7BQmdp3IXV3vIsAv4DxbPI/8k7DjU4gdy4ebUwj0szGye7Oy11NKWcaShGKMSQWuKOHzdcAdrumfgNhS1t8DXFzSMuUdJ9euJfHBv+HIzaX5q68QfvXVp5blFOYwdf1U3v/tfWLCY3j36neJbVTiP537fvscCk5S0Hkcn7x/mCs7RVG/zulLkGPyH6rc9pVSHqd3yqvzMg4HqbPnkDJ1KoGtW9Nq+jSC2rY9tTwhJYHHVz/OvhP7uPHCG5nUcxIh/iGV33FCPNRrycrsNhw/uYmxNfjek+r+iFmlimhCUaWynzjB4UceJevbbwkfdjVNn3sOW6izzlaBvYA3Nr/BnK1ziKoTxeyrZtO7aW/P7DjzKOz+FvpPZtH6w0SFB3Fp+0ae2bbyqpqUHGvKsVTlcWhCUSXK/f13Dt13PwWHDxP12GM0uPmmU5cE/5H2B4+vfpzfjv/GyLYjefjihwkLDCtji+WwdQkYB6ltRvHdNwf562Vtz3m6Yvxf+3puf0opj9CEos6R/vHHJD39DH7h4bR6Zz51evYEwO6wM3/7fGZsnEFYYBjTBk5jUMtBng8gIR6admfRgVAcBsb1qrmnu5SqSTSh1DITvpoAwNyhc89Z5sjP5+jz/yY9Pp46F19M81dexj8yEoCDJw7y+JrH2Zi8kcEtB/Nk3ydpGNzQ8wGm/A5HNmGG/JtFPx0krlUD2jSq6/n9KKU8ThOKAqAgMZFDkx4gd+tWIu68g0aTJiH+/hhjWPTHIv6z7j/4iz//7v9vhrcZ7n7plPJKiAfxY0vDK9mdspMpY9p4Zz9KKY/ThKLIWrWaww89hLHbiZ7xGmGDnUUOjp48ylM/PcWaw2vo27Qvz/Z7liahJVXT8RCHw3kzY9uBfLAtj5AAP67pqveeKFVdaEKpxYzDwbH/vsGx118nqEMHoqdNJTAmBmMMX+z9gud/fp5CRyGP936c6y+43nu9kiIH/gcZB8m//Ek++/gwV8c2oW6Q/ogqVV3ob2stc8Nr2wAo7J3G4X88zMlVq6g3cgRNnn4aW0gIablpPLf2OVbsX0G3Rt14vv/ztApvVTXBJcRDQChfFfYgM2+nFoJUqprRhFILBeTZ2TdmLIUpKTR5+mnqXz8eEeGHgz/w1E9PkZGfwaSek5jQuYI1uCqiIBe2fQydRrBw03FaNqxD79ZeGPRXSnmNJpRaJvREAQ1TcjHN6tHq/QWExMaSlZ/F//36fyzdtZQODTrw5pVvckHDC6o2sJ1fQ14Gya1H8tPPqTx4ZQdsNi+fYlNKeZQmlFokb+dOGqbkkhviR4clS/Bv0IBfk37lidVPkJSdxB2xd3B3t7sJ9Asse2Oetjke6jbhg5QYRPYwRu89Uara0YRSi6RMn46xQWpUCIVhIbz8yxTe2/EeLcNaMn/ofLo37m5NYNnHYedyzMV/ZdGGI/RrG0nz+h6oB6aUqlKaUGqJnC1byFzxDScaBLI/0vDaZ+PZm7GXGy64gcm9JlMnoI51wW37CBwFbG44hENpmfx9SBWfblNKeYQmlFoiZeo0/OrX57OLcvisRx6RBSd588o3uaTZJVaH5rz3pHEn5u8JIyw4hyGdvXivi1LKa6x6YqOqQid/+YWTa9aQ++dhLOtlp15IA5aOXOobyeT4Xjj4M7mdxvLltiRGdGtGcEAVXVmmlPIoTSg1nDGGlKnT8G/cmFdb/kaALYCYejGEB/rIs9kTPgSEr2UAuQUOxuljfpWqtjSh1HAnf/yRnA0bSL1hEOsyEmgW2gw/8ZEegDHOmxlj+jN/WwHtG9elW3Q9q6NSSlWQJpQazDgcJE+bRkCLaF5qvIGY8BgiQyKtDuu0xPVwfDdHW49kw4F0xsVFe7+8i1LKazSh1GCZy5eTt30H+8f25Y+Te5jUc5JvfWEnxIN/MAtOdMfPJozq0dzqiJRSlaAJpYYyhYWkTH+NgLZteKH+arpGduWKlldYHdZp9gLYugRHh6v5ICGDgRc0pnFYsNVRKaUqQRNKDZWx7FPy9+xh6+hYjuam8ECvB3yrd7JrJWSnsiViCCmZeYyL0zvjlaruNKHUQI78fI7NmEFAp478X50fGNB8ABc1ucjqsM6UEA8hDZl1uA0RoYEM6tjY6oiUUpWkCaUGSl+0iILDh/np2hiyCk4yqeckq0M6U24G/P4FuR1Hsfy3VEb1aE6An/4oKlXd6Z3yNYwjJ4djM2fi37Mr0/1/ZHjM8KqvHFyWHZ9CYS4r/AdSYDd6ukupGkL/LKxh0hYswJ5yjK+uisCB4Z4e91gd0rkS4qFhW/67sz5do+vRsYmP3GSplKoU7aHUIPbMTFLfmo307cVs2xpu7HgjzeueeSnu3KFzLYrOJSMR9q7iaK/J7FiTyXMjO1sbj1LKYzSh1CDH587DnpHBosv8qeNfhztj77Q6pHNtWQQYFub2IdDfwYhueu+JUjWFnvKqIQrT0jg+bx72yy9mkaxnQpcJNAhuYHVYZ3KVWnFEX8TcHcJVnaKoVyfA6qiUUh6iCaWGSJ31Fo7cXN7uk0NkSCQ3XXiT1SGd6+hWSN7OtshhpGcXaCFIpWoYTSg1QMHRo6S9/z45V1zMCtnB3d3utvaBWaXZvBBsAcw81pWm9YLp386H6ooppSpNE0oNcOyNNzB2O6/1OEpMeAyj24+2OqRzOeywZTG5ra/gy915jOkZjZ/Nh+7cV0pVmlsJRUT6iUioa/omEXlFRFp5NzTljvyDB0lfvIS0q3qxzu8g9/W4jwCbD45L7P0RspL4PmggDgNje+m9J0rVNO72UN4AskWkG/A3YDfwTkV3KiINRWSFiOx0vZc4eiwidhHZ5HotK/b5PBHZW2xZ94rGUt0dmzED8fPjP533ExsZy5WtrrQ6pJIlxGOCwnl1fxsujmlITGSo1REppTzM3YRSaIwxwEhghjHmdSCsEvt9BFhpjGkPrHTNlyTHGNPd9Rpx1rK/F1u2qRKxVFt5O3eSsexTEod2Y6f/MSb3muxbBSCL5J+EHZ9yrOXV/J5awFi9M16pGsndhJIpIo8CNwGfi4gNqMx5lZHAfNf0fGBUJbZVa6VMfw2pE8KUDn/Qr3k/3ysAWeS3LyA/iyWF/agT6Mc1sU2tjkgp5QXuJpTrgTzgdmNMEhANvFSJ/UYZY464ppOAqFLaBYvIOhFZKyJnJ53nRSRBRF4VkaBKxFIt5WzZSuaKFewcciFJ/ieZ3HOy1SGVLiEeR3g0M3Y3YlhsU0KD9H5apWoid3+zM4Fpxhi7iHQAOgIfnG8FEfkGaFLCoseLzxhjjIiYUjbTyhiTKCJtgG9FZIsxZjfwKM5EFAjMAh4Gni0ljonARICWLVueL+RqJWXaNKReOFPa7uCaNtf4XgHIIlnJsPtb/mh7G1nJhvF674lSNZa7PZQfgSARaQ4sB24G5p1vBWPMYGNMlxJenwBHRaQpgOs9uZRtJLre9wDfAz1c80eMUx4wF7j4PHHMMsbEGWPiGjVq5Obh+rbsX3/l5OrVbBzSmuxAw7097rU6pNJtXQLGzlsZccRE1OGiGB+7e18p5THuJhQxxmQD1wH/NcaMA7pUYr/LgFtd07cCn5yzQ5EGRaeyRCQS6Adsd80XJSPBOf6ytRKxVCvGGJKnToNGEbzcajvXX3D9OQUgfUpCPPmNYllyMIyxvaJ986IBpZRHuJ1QRKQvcCPweTnXLcmLwJUishMY7JpHROJEZLarzYXAOhHZDHwHvGiM2e5atkBEtgBbgEjgX5WIpVo5uWoVOevXs+qKxviH1OHOrj5YALJIyh9weCM/hV6BCFzXU6/uUqomc3cM5QGc4xZLjTHbXGMa31V0p8aYVOCKEj5fB9zhmv4JiC1l/UEV3Xd1ZhwOkqdOxTRtzH9b/sHdne+jYXBDq8MqXUI8Rmy8fKQr/dtF0qx+iNURKaW8yK2EYoz5AfhBROqKSF3XmMb93g1NnS1z+Qrytu/g8z+3oX6ocHOnm60OqXQOByR8SHqTfmzZG8xrw3QwXqmazt3SK7EishHYBmwXkfUiok9GqkLGbidl+nQKWzbl3Rb7fbcAZJGDayHjAJ9yKeHB/lzZqbQrw5VSNYW74yBvAg8aY1oZY1riLL/ylvfCUmfLWPYp+Xv2EH+p0KJeK67rcJ3VIZ1fQjwmIJRXDrZnZPfmBAf4WR2RUsrL3E0oocaYU2MmxpjvAS3GVEVMfj7HZswgt11zPok+yv097/fNApBFCnJh21L2NhpIemEg47TUilK1grsJZY+IPCkiMa7XE8AebwamTktbvJiCxETmXpJH58guXNXqKqtDOr+dyyE3g/kn+3BBVBixzetZHZFSqgq4m1BuAxoBH7lejVyfKS9z5OSQ+sZMTlwYzXfN0ny3AGRxCfEU1mnMe0dbMS5O7z1RqrZw9yqvNPSqLkukvf8+hSkpzBweTr/m/endtLfVIZ1f9nH442vWNx6LpPszqocP33SplPKo8yYUEfkUKK3OFiWUlFceZM/KInXWWxzr1oJ1TY6wqNcDVodUtu0fg6OA6Sk9GdSxMZF1a13dTqVqrbJ6KP+pkihUiY7PnYc9I4PpcfkMaz2Mjg07Wh1S2TbHkxXejjXJzXhLC0EqVaucN6G4bmhUFihMS+P4vHkcjItmV5NjvOLLBSCLHN8LB9eyvOEdRNYN4vILakYxTqWUe9waQ3HVzTr71FcGsA74l6uUivKg1Ldm48jJYVqPfK6/4M+0CKsGf+1vWQTAq0ndGN2vOQF+lSn3ppSqbtyt5fUlYAfed83fANTB+UySecC1Ho+sFis4mkzaggX8cVETjjXJ4s5YHy4AWcQYSIjncP04DiZFME5PdylV67ibUAYbY3oWm98iIhuMMT1F5CZvBFabHZv5Bg57IdN6HuUvXe4hIiTC6pDKdngDpO4iPuQ+urWoT4eoMKsjUkpVMXfPSfiJyKmHWInIRUBRLY1Cj0dVi+UfPEj6osVs7B2BvUkEt3a6teyVfMHmeBx+QcxN68q4XnpnvFK1kbs9lDuAt0WkLiDACeB2EQkFXvBWcLXRsRmvY2zCmz1S+X/dnvDtApBF7AWwdQnbw/qRlxfGtd2aWR2RUsoC7t7Y+CsQKyL1XPMZxRZ/6I3AaqO8XbvIWLaM1QMaULdZPcZ0GGN1SO7Z/S1kH2NWdhxDOjehXogP1xlTSnmNu+Xr64nIK8BKYKWIvFyUXJTnpEx/DUdwIPN6ZHB/Dx8vAFlcQjz5gfX5MreLFoJUqhZzdwzlbSATGO96nQDmeiuo2ihn6zYyly/n6z5BtGjRmatifLwAZJHcE/Db5/wYOIDG9cO4pG2k1REppSzi7hhKW2NM8fMvz4jIJm8EVFulTJtGYd1gFvY4ybSeD2CTanIPx45PoTCX14/HMeby5vjZtBCkUrWVu99aOSLSv2hGRPoBOd4JqfbJXreOk6tW8XFfG91jLqFvs75Wh+S+hHjSg6PZ6GjH2F5674lStZm7PZS7gfmucRMBjgN/8VZQtYkxhuRXp5JXvw4fd8vj3epQALJIRiJm7498HDCe3q0jaBlRDa5IU0p5jbtXeW0CuolIuGv+hFejqkVOrl5Nzvr1LBwaxBUdhtEpopPVIblv62IEw9ys3tw3VHsnStV2ZZWvf7CUzwEwxrzihZhqDWMMKa9O5WRkKN90K2Rp9/usDql8NsezP6QzxxzNGRbbxOpolFIWK6uHovUzvChz+Qpyt2/nneH+XNfpT7QIr0Z/5SdtheRtzHfcxvCuzagT6O7ZU6VUTVVW+fpnqiqQ2sbY7aRMn05ak1B+6Wbj865/tTqk8kmIxyH+LM2/mLf03hOlFO5f5XWKiGzwRiCM29SGAAAeQElEQVS1Tcann5K/ezdv983h1tgJ1aMAZBGHHbYsYkNgLxpENqVXqwZWR6SU8gEVudlBbzSoJJOfz7EZr5MUHcqurhHc0vkWq0Mqn32rIPMIb2f2Zkyv6FNjakqp2q0iCeVzj0dRy6QvWULBoUPMuSSHid3vIjQg1OqQymdzPHl+oXxnejKmp57uUko5lXsk1RjzhDcCqS0cubmk/PcN9sWEkNq1MeM6jLM6pPLJz8bsWMbXpg+9OzSnSb1gqyNSSvkId4tDZorIibNeB0VkqYi08XaQNUnagvexp6Twdv987ut5PwF+1aQAZJHfv0Dys3g/ty/j9M54pVQx7vZQpgKHcD4CWHA+ArgtsAFn4cjLvRFcTWPPyuLYrFnsaB+MdGvP0NZDrQ6p/BLiOe7fmN8llsGdGlsdjVLKh7g7hjLCGPOmMSbTGHPCGDMLGGKMiQf0Eh83HZ83H0dGBvP6FfBAr2pUALJIVgpm10oW5fdlRPdogvz9yl5HKVVruPuNli0i40XE5nqNB3Jdy4yXYqtRCtPSSJ07lw0XBhLVsy+XNLvE6pDKb+sSxNhZXNCPcXF6ukspdSZ3E8qNwM1AMnDUNX2TiIQA95Z3pyLSUERWiMhO13uJvRwRaSkiy0Vkh4hsF5EY1+etReRnEdklIvEiEljeGKpa6uzZOLJP8m5/Ow9UpwKQxSXEs9uvLX5RF9K5WbjV0SilfIxbCcUYs8cYc60xJtIY08g1vcsYk2OMWV2B/T4CrDTGtMf5FMhHSmn3DvCSMeZC4GKcCQ1gCvCqMaYdkAbcXoEYqkzB0WSOv/ceP3XxJzbuajpHdLY6pPI7thMOb+D93D6Mj2uh954opc7h7lVeHURkpYhsdc13FZHKXD48Epjvmp4PjCphn50Af2PMCgBjTJYxJluc32SDgMXnW9+XpL45E0dhAYv627ivRzUrAFkkIR4HNr6kH6N6NLc6GqWUD3L3lNdbwKNAAYAxJgHnlV4VFWWMOeKaTgKiSmjTAUgXkY9EZKOIvCQifkAEkG6MKXS1OwT47Ddc/qFDpMV/yMquwoDe42gZ3tLqkMrPGExCPD8TS9eOHWkY6vNnGJVSFnD3suE6xphfzjrNUVhaYwAR+QYoqab548VnjDFGREoa2PcHBgA9gANAPM6Hen3iZsxFcUwEJgK0bFn1X+bHZrxOoc3w+aUhLOh2V5Xv3yMOrEXSD/Bh/jWM00KQSqlSuJtQjolIW1xXdInIWODI+VYwxgwubZmIHBWRpsaYIyLSlNNjI8UdAjYZY/a41vkY6IPzvpf6IuLv6qVEA4nniWMWMAsgLi6uSq9Iy9u9m/Rln/BlHIzoO4HIkMiq3L3nJMSTJ8Gsr9OPlzo0sjoapZSPcveU1z3Am0BHEUkEHgAq8+f2MuBW1/StlNzr+BVn4ij6BhsEbDfGGOA7YGwZ61suZfp0CgKE7y+P4NZOt5a9gi8qzMOxbSlfFvbi6l7t8PerZvfOKKWqjLvfDonAXOB5YCGwgtMJoSJeBK4UkZ3AYNc8IhInIrMBjDF24CFgpYhswXmH/luu9R8GHhSRXTjHVOZUIhavyNm2jcyvl/NJnOHGvndRN7Cu1SFVzM7l2HLT+cjeX0utKKXOy91TXp8A6ThLrRyu7E6NManAFSV8vg64o9j8CqBrCe324LyM2GelTJ1Gdh0/NgxqxiPVrQBkMSYhnjSpz8nm/WnXuJomRaVUlXA3oUQbY6ph4SlrZK9fz8lVq/hooI07+04i0K+aXhWVk4b5/WuWFlzBmLgYq6NRSvk4d095/SQisV6NpIYwxnD0lVc4UdePvYM7cnXrq60OqeK2fYzNkc/ncinDuzW1OhqllI9zt4fSH/iLiOwF8nCOZxhjzDmno2q7k6vXkLt+A4uusnFPnwerXwHIYhybF7KX5rTs1Ifw4GpWZl8pVeXcTSjV+M/sqmOMIenVVzhW38aJIb2rZwHIImn7sB1cy5KC6xl/UTW8GVMpVeXcSijGmP3eDqQmyFyxgoLtO4i/xsb9vR+s3vWutiwC4Oe6g3ioTYTFwSilqoNyPwJYlczY7SRNfZXDETaCrxlCl8guVodUccZQsHEhGxwd6R/XE5utGidGpVSVqb4n+H3Mic8+w75nH/GX2rgvbpLV4VTO4Y0EpO3iI/sAxvbSUitKKfdoD8UDTH4+R6ZPY2+U0PLa8bQKb2V1SJViNi8knwCOtRhKi4Z1rA5HKVVNaA/FA9I/+giTeISPBgVzV4+7rQ6ncuwFFCYs5ht7D4b37mh1NEqpakQTSiU5cnM5MuM1fouGbtf+pfoWgCyy+zsCclP52nYZQzvrvSdKKfdpQqmktPc/QI4d59PB9ZjQ5Tarw6m0gk0LSTd1CY+9mpBAP6vDUUpVI5pQKsGelUXSzP+yqbUwaMS91bcAZJG8TOS3z/nU3ofRF7WxOhqlVDWjCaUSUufNw3Yii2+HRjH+gvFWh1N5Oz7F35HLL+GD6dmyvtXRKKWqGb3Kq4IK09JImTObXzsII4Y/WH0LQBaTve59UhyN6XTRldX7pkyllCW0h1JBKW+9Bbl5/DK8Dde0ucbqcCrvxGFCDq3mE9OfMXrviVKqArSHUgEFyckcf+9dVncS/nzNI9W6AGQRR8JibBgOtxxB4/Bgq8NRyuMKCgo4dOgQubm5Vofis4KDg4mOjiYgoGLFYDWhVEDSf2dgCgv5/boeTGzWz+pwPOLkugXscrTjsj59rA5FKa84dOgQYWFhxMTE6CndEhhjSE1N5dChQ7Ru3bpC26j+f1pXsfxDiZxYtIRvuwkTrnq0ZvxgHt1GWPpvfO13KVdcGGV1NEp5RW5uLhERETXjd9YLRISIiIhK9eA0oZTToekvY8dB8vjLiW1UM545lrv+AwqMH7YuYwj01x8JVXOVN5lc/+b/uP7N/3kpGt9T2WSr3x7lkLd7N7mffsXyXn7cMegfVofjGQ479s3x/ODoyjV9a0aCVMpX1a17+l61oUOHUr9+fYYPH15i23vuuYfu3bvTqVMnQkJC6N69O927d2fx4sXl2ueGDRv46quvKhW3u3QMpRwOvPJ/5AYYCm8cQUy9GKvD8Yx9qwnNS+bX8Ak82qye1dEoVWv8/e9/Jzs7mzfffLPE5a+//joA+/btY/jw4WzatKlC+9mwYQNbt25l6NChFY7VXdpDcVPu9u0UrvyRr3sHcvuAyVaH4zFpa98j04QQ3Xu01aEoVatcccUVhIWFVWjdnTt3MmTIEHr16sWll17KH3/8AcDChQvp0qUL3bp1Y+DAgeTk5PDss8+yYMGCCvVuykt7KG6Y8NUERs3cRrNgqHfrTTSq08jqkDwjP5s6uz7jU9Ob4b3aWh2NUlXmmU+3sf3wiTLbbT/ibOPOOEqnZuE8dW3nSsfmjokTJzJ79mzatm3LmjVruPfee1m+fDnPPPMM33//PVFRUaSnpxMSEsI///lPtm7dytSpU70elyYUN9zyf5tpcjiHJYPrMOniu6wOx2MKd3xBkCObQy2upUFo9b/TX6naID09nbVr1zJmzJhTnxUWFgLQr18/brnlFsaNG8d1111X5bFpQnFDYFYeaaEQc9v/IyywYl1UX3T8f+9QaBrSrX/Jg4JK1VTu9iSKeibxf+3rzXDKxRhDZGRkiWMqb731Fj///DOfffYZPXv2ZOPGjVUam46huGHeYBvvXeHH+G43Wx2K55w8RkTSKlb4XcqADo2tjkYp5aYGDRrQtGlTli5dCoDD4WDz5s0A7Nmzhz59+vDcc8/RoEEDEhMTCQsLIzMzs0pi04Tihux+XUnu37FGFIAskrluIX44KOwyHn8//TFQqqoNGDCAcePGsXLlSqKjo/n666/dXnfhwoXMnDmTbt260blzZz777DMAJk+eTGxsLLGxsQwcOJAuXbowaNAgNm/eTI8ePXRQ3hcE+gXWqGQCkL3uAw44WjFwwGVWh6JUrZGVlXVqetWqVW6tExMTw9atW8/4rE2bNiUmoGXLlp3zWaNGjVi3bl05I60YTSi1kDm2k6jMrXwZfgd/aVTNHwqmlBf50thJdaAJxQ1zh861OgTPmXsNSSnHaGyE+hf/yepolFI1iJ48r22MISg7ibV04YqLu1kdjVKqBtGEUsvk5WbTkHT2Nx9OWHDFnnmglFIl0YRSy/yRnEm2CaLtpX+2OhSlVA1jSUIRkYYiskJEdrreG5TSrqWILBeRHSKyXURiXJ/PE5G9IrLJ9epelfFXV2b3d7Q1B1jliOWiDi2sDkcp3zf3GudLucWqHsojwEpjTHtgpWu+JO8ALxljLgQuBpKLLfu7Maa761WxMpy1hcNB7sopmHdHc9A05ndbG2w2fciQUlWtqsvXL126lJdeeqnScbvLqqu8RgKXu6bnA98DDxdvICKdAH9jzAoAY0wWqvxy0jjx/m2EH/yWT+z92GlrzdDAzVZHpVSt56ny9YWFhfj7l/xVPnp01VYRt6qHEmWMOeKaTgJKeu5sByBdRD4SkY0i8pKI+BVb/ryIJIjIqyIS5PWIqyFzeCNZ0y8h+MAP/Mf/Tlrc8R4Pxeyjiz73RCnLVaZ8ff/+/Zk8eTJxcXHMmDGDTz75hN69e9OjRw+uuuoqkpOdJ3Nmz57NAw88AMBNN93EpEmTuOSSS2jTps2p0i2e5LUeioh8AzQpYdHjxWeMMUZETAnt/IEBQA/gABAP/AWYAzyKMxEFArNw9m6eLSWOicBEgJYtW1bgSKohY8j9ZR5+X/2dE44wZjd9lftu/pNWFFaqyJePQNKWstslJTjf3RlHaRILV79YubjKwW63n7oDPi0tjREjRiAizJw5k5dffpkpU6acs05ycjJr1qxhy5YtjB8/3uM9GK8lFGPM4NKWichREWlqjDkiIk05c2ykyCFgkzFmj2udj4E+wJxivZs8EZkLPHSeOGbhTDrExcWVlLhqloIcMhZPot7v8axyxPJHv1d54sq402MmEz63Nj6llEdcf/31p6YPHDjA+PHjSUpKIi8vjw4dOpS4zqhRoxARunbtSmJiosdjsmoMZRlwK/Ci6/2TEtr8CtQXkUbGmBRgELAOoFgyEmAUsLWE9Wuf43tIn3cD9U/8ziwZR5ebn+f29iWdTVSqlnO3J1HUM/HBP8RCQ0NPTd9zzz089thjDBs2jG+++YYXXyz5+IKCTo8OGOP5v6+tSigvAh+KyO3AfmA8gIjEAXcZY+4wxthF5CFgpStxrAfecq2/QEQaAQJsAmrOU68qKH/rp9g/ugvshhcaPsvtE/5K4/Bgq8NSSlWBjIwMmjdvjjGG+fPnWxaHJQnFGJMKXFHC5+uAO4rNrwC6ltBukFcDrE7shaR//hT1N8wgwdGan3q+wt+vvVxL0ivl4wYMGMBvv/1GVlYW0dHRzJkzhyFDhlRoW08//TSjR4+mYcOGXH755Rw5cqTslbxAvNHt8VVxcXGmqso4V4msZFLn30REys8sZjCR417l8s615MIDpcppx44dXHjhheVbyYdPeXlLSf+fRGS9MSaurHW12nA1VbD3J3Lev5nQ/AymhU9mzG3/ILpBHavDUqpmqUWJxBM0oVQ3xpDx3XRCf3yG445Ivuw0m7vHjiDQX09xKaWspQmlOsnLJPm9O2l88Eu+NXHkj3idu+M6Wh2VUkoBmlCqDfvRHWTMu56I7APMCbmVgbf9izaNw60OSymlTtGEUg2c+OUDAr98ALsjkFmtp/KXP99MSKBf2SsqpVQV0oTiywrzSVr0N5r8/g7rzQUkXvkGd/fvZXVUStUaE76aANSwx4B7kY7k+ihH2kGSpg+iye/vsChgBHXu/JIRmkyUqtaKytdv2rSJvn370rlzZ7p27Up8fPw5bT1Rvh5gw4YNfPXVVx6JvyzaQ/FBWdtXYBbfTl17HrObPs31f7lXH9erVA1Sp04d3nnnHdq3b8/hw4fp1asXQ4YMoX79+qfauFu+viwbNmxg69atDB061COxn4/2UHyJw8GRT5+jzofjSLLXZXm/hdz+1wc0mShVw3To0IH27dsD0KxZMxo3bkxKSorb6+/cuZMhQ4bQq1cvLr30Uv744w8AFi5cSJcuXejWrRsDBw4kJyeHZ599lgULFlSod1Ne2kPxESY7jUNv30yLY6tYbhtA1I0zua5ttNVhKVUjTfllCr8d/63MdkVtisZSzqdjw448fPHDZbY72y+//EJ+fj5t27Z1e52JEycye/Zs2rZty5o1a7j33ntZvnw5zzzzDN9//z1RUVGkp6cTEhLCP//5T7Zu3crUqVPLHVt5aULxAdn715P97p+JKkjh3Yh7GX7bP2lQV58ZplRNd+TIEW6++Wbmz5+PzebeCaP09HTWrl3LmDFjTn1WWFgIQL9+/bjlllsYN24c1113nVdiPh9NKBY78t0sIn54jHwTxrJec7hx+Ch93rtSXuZuT8KbV3mdOHGCa665hueff54+ffq4vZ4xhsjIyBLHVN566y1+/vlnPvvsM3r27MnGjRs9GXKZdAzFKgU57J3zF5r+8Hc2SCcOX/81Y0eM1mSiVC2Qn5/P6NGjueWWWxg7dmy51m3QoAFNmzY99Qhfh8PB5s2bAdizZw99+vThueeeo0GDBiQmJhIWFkZmZqbHj6EkmlAskJe8i8SXB9D64FKW1P0zbR74krhOJT9hTSlV83z44Yf8+OOPzJs379TlwOW5imvhwoXMnDmTbt260blzZz777DMAJk+eTGxsLLGxsQwcOJAuXbowaNAgNm/eTI8ePbw+KK/l66vY0V8/IvTzeyk0sPLCZxk57jZ9dolSVaAi5etr442NWr6+OrAXsuvDR2n3+yy20YYT185hTFxPq6NSSp1HbUoknqAJpQrkpyeROPtPtMvawNfBV9PljjfoHNnA6rCUUsqjNKF4Wcr2H7AtnkBT+wk+jnmcYTc/pM8uUUrVSJpQvMUYdn76EjEbXuSIiWT74A8ZNWCQ1VEppZTXaELxgsLsDHbNuY2Oqd/wP//eNJswjwHNm1kdllJKeZUmFA9L3beZ3PdupH3BIb5ocheDbv8XwYFai0up6mj/zbcA0OrddyyOpHrQk/ke9Mc3cwmZdxVBBSdYfckcht09RZOJUuqUqi5fv3TpUl566SWPxV8W7aF4gKMgj63z7qNrYjxbbBcS8uf5XNbuAqvDUkr5KE+Wry8sLMTfv+Sv8tGjR3s++PPQhFJJ6Un7OPb2DXTN38G39cdy8cQZ1K0TYnVYSikf1qHD6coYxcvXF08o59O/f38uuugiVq1axU033UTr1q3597//TX5+Po0aNeK9996jcePGzJ49+1Sl4ZtuuomIiAh+/fVXkpKSePnllz2ecDShVMKutZ8R8dXdNDH5/ND9JQaOuhMRrcWllK9L+ve/ydtRdvn63N+cbYrGUs4n6MKONHnssXLHUpHy9QB2u52iyh9paWmMGDECEWHmzJm8/PLLTJky5Zx1kpOTWbNmDVu2bGH8+PGaUHyBcdjZuOBJuu/6L/tt0eSOmcdlsWVWJVBKqTNUpHx9keuvv/7U9IEDBxg/fjxJSUnk5eWd0QMqbtSoUYgIXbt2JTExsVKxl0QTSjllpqewb9ZN9Mxey9q6A+l459vUr9/Q6rCUUuXgbk/Cm1d5VbR8fZHQ0NBT0/fccw+PPfYYw4YN45tvvuHFF18scZ2goNPPWfJGHUdNKOWwN2ENQUsncIHjGD92eJj+NzyCTQs7KqXKqTLl60uSkZFB8+bNMcYwf/58D0RYMfpt6KZfP5pKsyUj8TOF7Bz2IZfe+JgmE6VUhVS2fP3Znn76aUaPHs1FF11EVFSUByMtHy1fXwbjcPDLjFvpfXwZW4J60PS2BURGNfdShEopb6lI+fraeGOjlq/3IrHZMA3b8r86t3HxX17Cr5TrvZVSNU9tSiSeoN+Obuhz09NWh6CUUj7PkkEAEWkoIitEZKfr/ZyHg4jIQBHZVOyVKyKjXMtai8jPIrJLROJFJLDqj0IppVRxVo0qPwKsNMa0B1a65s9gjPnOGNPdGNMdGARkA8tdi6cArxpj2gFpwO1VE7ZSqjqrTWPGFVHZ/z9WJZSRQNG1bfOBUWW0Hwt8aYzJFuet6IOAogpp7qyvlKrlgoODSU1N1aRSCmMMqampBAcHV3gbVo2hRBljjrimk4CyrnO7AXjFNR0BpBtjCl3zhwC97EopdV7R0dEcOnSIlJQUq0PxWcHBwURHR1d4fa8lFBH5BmhSwqLHi88YY4yIlPong4g0BWKBrysYx0RgIkDLli0rsgmlVA0QEBBA69atrQ6jRvNaQjHGDC5tmYgcFZGmxpgjroSRfJ5NjQeWGmMKXPOpQH0R8Xf1UqKBUovSGGNmAbPAeR9KeY9DKaWUe6waQ1kG3OqavhX45Dxt/wR8UDRjnCdAv8M5ruLO+koppaqAVQnlReBKEdkJDHbNIyJxIjK7qJGIxAAtgB/OWv9h4EER2YVzTGVOFcSslFLqPGpV6RURSQH2V3D1SOCYB8OxUk05lppyHKDH4qtqyrFU9jhaGWMaldWoViWUyhCRde7UsqkOasqx1JTjAD0WX1VTjqWqjkPL5SqllPIITShKKaU8QhOK+2ZZHYAH1ZRjqSnHAXosvqqmHEuVHIeOoSillPII7aEopZTyCE0o5SAiz4lIgquc/nIRaWZ1TBUlIi+JyG+u41kqIvWtjqkiRGSciGwTEYeIVMurcURkqIj87nocwzmVt6sLEXlbRJJFZKvVsVSGiLQQke9EZLvrZ2uS1TFVlIgEi8gvIrLZdSzPeHV/esrLfSISbow54Zq+H+hkjLnL4rAqRESuAr41xhSKyBQAY8zDFodVbiJyIeAA3gQeMsaU7xnPFhMRP+AP4EqchU5/Bf5kjNluaWAVICKXAlnAO8aYLlbHU1GuclBNjTEbRCQMWA+Mqqb/JgKEGmOyRCQAWA1MMsas9cb+tIdSDkXJxCUUqLbZ2BizvFjF5rU4a6JVO8aYHcaY362OoxIuBnYZY/YYY/KBhTgf71DtGGN+BI5bHUdlGWOOGGM2uKYzgR1U04rmxinLNRvgennte0sTSjmJyPMichC4Efin1fF4yG3Al1YHUUs1Bw4Wm9fHMfgQV/mnHsDP1kZScSLiJyKbcBbhXWGM8dqxaEI5i4h8IyJbS3iNBDDGPG6MaQEsAO61NtrzK+tYXG0eBwpxHo9Pcuc4lPI0EakLLAEeOOvsRLVijLG7nnwbDVwsIl47HWnVA7Z81vnK7p9lAfAF8JQXw6mUso5FRP4CDAeuMD48mFaOf5PqKBFnAdQi530cg6oarvGGJcACY8xHVsfjCcaYdBH5DhgKeOXCCe2hlIOItC82OxL4zapYKktEhgL/AEYYY7KtjqcW+xVoLyKtRSQQ59NJl1kcU63mGsieA+wwxrxSVntfJiKNiq7gFJEQnBd/eO17S6/yKgcRWQJcgPOqov3AXcaYavnXpKv0fxDOB5YBrK2OV6yJyGjgNaARkA5sMsYMsTaq8hGRYcBUwA942xjzvMUhVYiIfABcjrOy7VHgKWNMtXu0hIj0B1YBW3D+rgM8Zoz5wrqoKkZEugLzcf5s2YAPjTHPem1/mlCUUkp5gp7yUkop5RGaUJRSSnmEJhSllFIeoQlFKaWUR2hCUUop5RGaUJTyIBHJKrvVeddfLCJtXNN1ReRNEdktIutF5HsR6S0igSLyo4jojcnKp2hCUcpHiEhnwM8Ys8f10WycxRbbG2N6AROASFcRyZXA9dZEqlTJNKEo5QXi9JKr5tgWEbne9blNRP7rehbNChH5QkTGula7EfjE1a4t0Bt4whjjADDG7DXGfO5q+7GrvVI+Q7vMSnnHdUB3oBvOO8d/FZEfgX5ADNAJaIyzNPrbrnX6AR+4pjvjvOvfXsr2twIXeSVypSpIeyhKeUd/4ANXpdejwA84E0B/YJExxmGMSQK+K7ZOUyDFnY27Ek2+6wFQSvkETShK+Y4cINg1vQ3o5nqiY2mCgFyvR6WUmzShKOUdq4DrXQ83agRcCvwCrAHGuMZSonAWUyyyA2gHYIzZDawDnnFVv0VEYkTkGtd0BHDMGFNQVQekVFk0oSjlHUuBBGAz8C3wD9cpriU4n8q4HXgP2ABkuNb5nDMTzB1AFLBLRLYC83A+dQ9goKu9Uj5Dqw0rVcVEpK4xJsvVy/gF6GeMSXI9r+I713xpg/FF2/gIeMQY80cVhKyUW/QqL6Wq3meuhx4FAs+5ei4YY3JE5Cmcz5Q/UNrKrgdxfazJRPka7aEopZTyCB1DUUop5RGaUJRSSnmEJhSllFIeoQlFKaWUR2hCUUop5RGaUJRSSnnE/wd7snjHsyui+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#pd.DataFrame(grid.cv_results_).to_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#cvresult = pd.DataFrame.from_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#test_means = cv_results['mean_test_score']\n",
    "#test_stds = cv_results['std_test_score'] \n",
    "#train_means = cvresult['mean_train_score']\n",
    "#train_stds = cvresult['std_train_score'] \n",
    "\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'neg-logloss' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上图给出了L1正则和L2正则下、不同正则参数C对应的模型在训练集上测试集上的正确率（score）。可以看出在训练集上C越大（正则越少）的模型性能越好；\n",
    "但在测试集上当C=0.1时性能最好（L1正则和L2正则均是）"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
